require './my_text'
require './n_gram_counter'
require './n_gram'

NGramCounter.test
NGram.test

origin_stdout = $stdout

MAX_ARITY = 3

counter = NGramCounter.new MAX_ARITY
text = MyText.new "./ru"

text.sentences.each { |sentence|
  ns = (1..MAX_ARITY).collect { |arity| NGram.new arity }
  sentence.each { |word|
    ns.each { |ng|
      ng << word
      next unless ng.ready?
      counter << ng
    }
  }
}

$stdout = File.new "./ru.lm", "w"
counter.print
$stdout = origin_stdout

return

ROUTE = ARGV[0] || "../data/en/"

Dir.foreach(ROUTE) { |dir|
  next if dir == "1989"
  root_dir = File.join ROUTE, dir
  next unless File.directory? root_dir
  next if ['.', '..'].include? dir

  counter = NGramCounter.new MAX_ARITY

  puts "== Entering "+root_dir
  sub = 0
  Dir.foreach(root_dir) { |data_file|
    file_url = File.join root_dir, data_file
    next unless File.extname(file_url) == ".xml"
    next if File.size?(file_url) > 100000
    puts "==== Scaning "+data_file+" ("+File.size?(file_url).to_s+")"

    text = MyText.new file_url

    text.sentences.each { |sentence|
      ns = (1..MAX_ARITY).collect { |arity| NGram.new arity }
      sentence.each { |word|
        ns.each { |ng|
          ng << word
          next unless ng.ready?
          counter << ng
        }
      }
    }
    if counter.total > 1000000
      puts "---dump"
      $stdout = File.new "./ngrams/"+dir+"_"+sub.to_s, "w"
      counter.print
      $stdout = origin_stdout
      sub += 1
      puts counter.total
      counter = NGramCounter.new MAX_ARITY
    end
  }
  if counter.total > 200000
    $stdout = File.new "./ngrams/"+dir+"_"+sub.to_s, "w"
    counter.print
    $stdout = origin_stdout
  end
  puts counter.total
}
